package org.apache.lucene.store;

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.RandomAccessFile;

import java.util.Collection;
import static java.util.Collections.synchronizedSet;

import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.Future;

import org.apache.lucene.util.ThreadInterruptedException;
import org.apache.lucene.util.Constants;

/**
 * <a name="subclasses"/>
 * Base class for Directory implementations that store index
 * files in the file system.  There are currently three core
 * subclasses:
 *
 * <ul>
 *
 *  <li> {@link SimpleFSDirectory} is a straightforward
 *       implementation using java.io.RandomAccessFile.
 *       However, it has poor concurrent performance
 *       (multiple threads will bottleneck) as it
 *       synchronizes when multiple threads read from the
 *       same file.
 *
 *  <li> {@link NIOFSDirectory} uses java.nio's
 *       FileChannel's positional io when reading to avoid
 *       synchronization when reading from the same file.
 *       Unfortunately, due to a Windows-only <a
 *       href="http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6265734">Sun
 *       JRE bug</a> this is a poor choice for Windows, but
 *       on all other platforms this is the preferred
 *       choice. Applications using {@link Thread#interrupt()} or
 *       {@link Future#cancel(boolean)} should use
 *       {@link SimpleFSDirectory} instead. See {@link NIOFSDirectory} java doc
 *       for details.
 *        
 *        
 *
 *  <li> {@link MMapDirectory} uses memory-mapped IO when
 *       reading. This is a good choice if you have plenty
 *       of virtual memory relative to your index size, eg
 *       if you are running on a 64 bit JRE, or you are
 *       running on a 32 bit JRE but your index sizes are
 *       small enough to fit into the virtual memory space.
 *       Java has currently the limitation of not being able to
 *       unmap files from user code. The files are unmapped, when GC
 *       releases the byte buffers. Due to
 *       <a href="http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4724038">
 *       this bug</a> in Sun's JRE, MMapDirectory's {@link IndexInput#close}
 *       is unable to close the underlying OS file handle. Only when
 *       GC finally collects the underlying objects, which could be
 *       quite some time later, will the file handle be closed.
 *       This will consume additional transient disk usage: on Windows,
 *       attempts to delete or overwrite the files will result in an
 *       exception; on other platforms, which typically have a &quot;delete on
 *       last close&quot; semantics, while such operations will succeed, the bytes
 *       are still consuming space on disk.  For many applications this
 *       limitation is not a problem (e.g. if you have plenty of disk space,
 *       and you don't rely on overwriting files on Windows) but it's still
 *       an important limitation to be aware of. This class supplies a
 *       (possibly dangerous) workaround mentioned in the bug report,
 *       which may fail on non-Sun JVMs.
 *       
 *       Applications using {@link Thread#interrupt()} or
 *       {@link Future#cancel(boolean)} should use
 *       {@link SimpleFSDirectory} instead. See {@link MMapDirectory}
 *       java doc for details.
 * </ul>
 *
 * Unfortunately, because of system peculiarities, there is
 * no single overall best implementation.  Therefore, we've
 * added the {@link #open} method, to allow Lucene to choose
 * the best FSDirectory implementation given your
 * environment, and the known limitations of each
 * implementation.  For users who have no reason to prefer a
 * specific implementation, it's best to simply use {@link
 * #open}.  For all others, you should instantiate the
 * desired implementation directly.
 *
 * <p>The locking implementation is by default {@link
 * NativeFSLockFactory}, but can be changed by
 * passing in a custom {@link LockFactory} instance.
 *
 * @see Directory
 */
public abstract class FSDirectory extends Directory {

    /**
     * Default read chunk size.  This is a conditional default: on 32bit JVMs, it defaults to 100 MB.  On 64bit JVMs, it's
     * <code>Integer.MAX_VALUE</code>.
     *
     * @see #setReadChunkSize
     */
    public static final int DEFAULT_READ_CHUNK_SIZE = Constants.JRE_IS_64BIT ? Integer.MAX_VALUE : 100 * 1024 * 1024;

    protected final File directory; // The underlying filesystem directory
    protected final Set<String> staleFiles = synchronizedSet(new HashSet<String>()); // Files written, but not yet sync'ed
    private int chunkSize = DEFAULT_READ_CHUNK_SIZE; // LUCENE-1566

    // returns the canonical version of the directory, creating it if it doesn't exist.
    private static File getCanonicalPath(File file) throws IOException {
        return new File(file.getCanonicalPath());
    }

    /** Create a new FSDirectory for the named location (ctor for subclasses).
     * @param path the path of the directory
     * @param lockFactory the lock factory to use, or null for the default
     * ({@link NativeFSLockFactory});
     * @throws IOException
     */
    protected FSDirectory(File path, LockFactory lockFactory) throws IOException {
        // new ctors use always NativeFSLockFactory as default:
        if (lockFactory == null) {
            lockFactory = new NativeFSLockFactory();
        }
        directory = getCanonicalPath(path);

        if (directory.exists() && !directory.isDirectory())
            throw new NoSuchDirectoryException("file '" + directory + "' exists but is not a directory");

        setLockFactory(lockFactory);
    }

    /** Creates an FSDirectory instance, trying to pick the
     *  best implementation given the current environment.
     *  The directory returned uses the {@link NativeFSLockFactory}.
     *
     *  <p>Currently this returns {@link MMapDirectory} for most Solaris
     *  and Windows 64-bit JREs, {@link NIOFSDirectory} for other
     *  non-Windows JREs, and {@link SimpleFSDirectory} for other
     *  JREs on Windows. It is highly recommended that you consult the
     *  implementation's documentation for your platform before
     *  using this method.
     *
     * <p><b>NOTE</b>: this method may suddenly change which
     * implementation is returned from release to release, in
     * the event that higher performance defaults become
     * possible; if the precise implementation is important to
     * your application, please instantiate it directly,
     * instead. For optimal performance you should consider using
     * {@link MMapDirectory} on 64 bit JVMs.
     *
     * <p>See <a href="#subclasses">above</a> */
    public static FSDirectory open(File path) throws IOException {
        return open(path, null);
    }

    /** Just like {@link #open(File)}, but allows you to
     *  also specify a custom {@link LockFactory}. */
    public static FSDirectory open(File path, LockFactory lockFactory) throws IOException {
        if ((Constants.WINDOWS || Constants.SUN_OS || Constants.LINUX) && Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
            return new MMapDirectory(path, lockFactory);
        } else if (Constants.WINDOWS) {
            return new SimpleFSDirectory(path, lockFactory);
        } else {
            return new NIOFSDirectory(path, lockFactory);
        }
    }

    @Override
    public void setLockFactory(LockFactory lockFactory) throws IOException {
        super.setLockFactory(lockFactory);

        // for filesystem based LockFactory, delete the lockPrefix, if the locks are placed
        // in index dir. If no index dir is given, set ourselves
        if (lockFactory instanceof FSLockFactory) {
            final FSLockFactory lf = (FSLockFactory) lockFactory;
            final File dir = lf.getLockDir();
            // if the lock factory has no lockDir set, use the this directory as lockDir
            if (dir == null) {
                lf.setLockDir(directory);
                lf.setLockPrefix(null);
            } else if (dir.getCanonicalPath().equals(directory.getCanonicalPath())) {
                lf.setLockPrefix(null);
            }
        }

    }

    /** Lists all files (not subdirectories) in the
     *  directory.  This method never returns null (throws
     *  {@link IOException} instead).
     *
     *  @throws NoSuchDirectoryException if the directory
     *   does not exist, or does exist but is not a
     *   directory.
     *  @throws IOException if list() returns null */
    public static String[] listAll(File dir) throws IOException {
        if (!dir.exists())
            throw new NoSuchDirectoryException("directory '" + dir + "' does not exist");
        else if (!dir.isDirectory())
            throw new NoSuchDirectoryException("file '" + dir + "' exists but is not a directory");

        // Exclude subdirs
        String[] result = dir.list(new FilenameFilter() {
            public boolean accept(File dir, String file) {
                return !new File(dir, file).isDirectory();
            }
        });

        if (result == null)
            throw new IOException("directory '" + dir + "' exists and is a directory, but cannot be listed: list() returned null");

        return result;
    }

    /** Lists all files (not subdirectories) in the
     * directory.
     * @see #listAll(File) */
    @Override
    public String[] listAll() throws IOException {
        ensureOpen();
        return listAll(directory);
    }

    /** Returns true iff a file with the given name exists. */
    @Override
    public boolean fileExists(String name) {
        ensureOpen();
        File file = new File(directory, name);
        return file.exists();
    }

    /** Returns the time the named file was last modified. */
    @Override
    public long fileModified(String name) {
        ensureOpen();
        File file = new File(directory, name);
        return file.lastModified();
    }

    /** Returns the time the named file was last modified. */
    public static long fileModified(File directory, String name) {
        File file = new File(directory, name);
        return file.lastModified();
    }

    /** Set the modified time of an existing file to now.
     *  @deprecated Lucene never uses this API; it will be
     *  removed in 4.0. */
    @Override
    @Deprecated
    public void touchFile(String name) {
        ensureOpen();
        File file = new File(directory, name);
        file.setLastModified(System.currentTimeMillis());
    }

    /** Returns the length in bytes of a file in the directory. */
    @Override
    public long fileLength(String name) throws IOException {
        ensureOpen();
        File file = new File(directory, name);
        final long len = file.length();
        if (len == 0 && !file.exists()) {
            throw new FileNotFoundException(name);
        } else {
            return len;
        }
    }

    /** Removes an existing file in the directory. */
    @Override
    public void deleteFile(String name) throws IOException {
        ensureOpen();
        File file = new File(directory, name);
        if (!file.delete())
            throw new IOException("Cannot delete " + file);
        staleFiles.remove(name);
    }

    /** Creates an IndexOutput for the file with the given name. */
    @Override
    public IndexOutput createOutput(String name) throws IOException {
        ensureOpen();

        ensureCanWrite(name);
        return new FSIndexOutput(this, name);
    }

    protected void ensureCanWrite(String name) throws IOException {
        if (!directory.exists())
            if (!directory.mkdirs())
                throw new IOException("Cannot create directory: " + directory);

        File file = new File(directory, name);
        if (file.exists() && !file.delete()) // delete existing, if any
            throw new IOException("Cannot overwrite: " + file);
    }

    protected void onIndexOutputClosed(FSIndexOutput io) {
        staleFiles.add(io.name);
    }

    @Deprecated
    @Override
    public void sync(String name) throws IOException {
        sync(Collections.singleton(name));
    }

    @Override
    public void sync(Collection<String> names) throws IOException {
        ensureOpen();
        Set<String> toSync = new HashSet<String>(names);
        toSync.retainAll(staleFiles);

        for (String name : toSync)
            fsync(name);

        staleFiles.removeAll(toSync);
    }

    // Inherit javadoc
    @Override
    public IndexInput openInput(String name) throws IOException {
        ensureOpen();
        return openInput(name, BufferedIndexInput.BUFFER_SIZE);
    }

    @Override
    public String getLockID() {
        ensureOpen();
        String dirName; // name to be hashed
        try {
            dirName = directory.getCanonicalPath();
        } catch (IOException e) {
            throw new RuntimeException(e.toString(), e);
        }

        int digest = 0;
        for (int charIDX = 0; charIDX < dirName.length(); charIDX++) {
            final char ch = dirName.charAt(charIDX);
            digest = 31 * digest + ch;
        }
        return "lucene-" + Integer.toHexString(digest);
    }

    /** Closes the store to future operations. */
    @Override
    public synchronized void close() {
        isOpen = false;
    }

    /** @deprecated Use {@link #getDirectory} instead. */
    @Deprecated
    public File getFile() {
        return getDirectory();
    }

    /** @return the underlying filesystem directory */
    public File getDirectory() {
        ensureOpen();
        return directory;
    }

    /** For debug output. */
    @Override
    public String toString() {
        return this.getClass().getName() + "@" + directory + " lockFactory=" + getLockFactory();
    }

    /**
     * Sets the maximum number of bytes read at once from the
     * underlying file during {@link IndexInput#readBytes}.
     * The default value is {@link #DEFAULT_READ_CHUNK_SIZE};
     *
     * <p> This was introduced due to <a
     * href="http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6478546">Sun
     * JVM Bug 6478546</a>, which throws an incorrect
     * OutOfMemoryError when attempting to read too many bytes
     * at once.  It only happens on 32bit JVMs with a large
     * maximum heap size.</p>
     *
     * <p>Changes to this value will not impact any
     * already-opened {@link IndexInput}s.  You should call
     * this before attempting to open an index on the
     * directory.</p>
     *
     * <p> <b>NOTE</b>: This value should be as large as
     * possible to reduce any possible performance impact.  If
     * you still encounter an incorrect OutOfMemoryError,
     * trying lowering the chunk size.</p>
     */
    public final void setReadChunkSize(int chunkSize) {
        // LUCENE-1566
        if (chunkSize <= 0) {
            throw new IllegalArgumentException("chunkSize must be positive");
        }
        if (!Constants.JRE_IS_64BIT) {
            this.chunkSize = chunkSize;
        }
    }

    /**
     * The maximum number of bytes to read at once from the
     * underlying file during {@link IndexInput#readBytes}.
     * @see #setReadChunkSize
     */
    public final int getReadChunkSize() {
        // LUCENE-1566
        return chunkSize;
    }

    protected static class FSIndexOutput extends BufferedIndexOutput {
        private final FSDirectory parent;
        private final String name;
        private final RandomAccessFile file;
        private volatile boolean isOpen; // remember if the file is open, so that we don't try to close it more than once

        public FSIndexOutput(FSDirectory parent, String name) throws IOException {
            this.parent = parent;
            this.name = name;
            file = new RandomAccessFile(new File(parent.directory, name), "rw");
            isOpen = true;
        }

        /** output methods: */
        @Override
        public void flushBuffer(byte[] b, int offset, int size) throws IOException {
            file.write(b, offset, size);
        }

        @Override
        public void close() throws IOException {
            parent.onIndexOutputClosed(this);
            // only close the file if it has not been closed yet
            if (isOpen) {
                boolean success = false;
                try {
                    super.close();
                    success = true;
                } finally {
                    isOpen = false;
                    if (!success) {
                        try {
                            file.close();
                        } catch (Throwable t) {
                            // Suppress so we don't mask original exception
                        }
                    } else {
                        file.close();
                    }
                }
            }
        }

        /** Random-access methods */
        @Override
        public void seek(long pos) throws IOException {
            super.seek(pos);
            file.seek(pos);
        }

        @Override
        public long length() throws IOException {
            return file.length();
        }

        @Override
        public void setLength(long length) throws IOException {
            file.setLength(length);
        }
    }

    protected void fsync(String name) throws IOException {
        File fullFile = new File(directory, name);
        boolean success = false;
        int retryCount = 0;
        IOException exc = null;
        while (!success && retryCount < 5) {
            retryCount++;
            RandomAccessFile file = null;
            try {
                try {
                    file = new RandomAccessFile(fullFile, "rw");
                    file.getFD().sync();
                    success = true;
                } finally {
                    if (file != null)
                        file.close();
                }
            } catch (IOException ioe) {
                if (exc == null)
                    exc = ioe;
                try {
                    // Pause 5 msec
                    Thread.sleep(5);
                } catch (InterruptedException ie) {
                    throw new ThreadInterruptedException(ie);
                }
            }
        }
        if (!success)
            // Throw original exception
            throw exc;
    }
}
